package com.joe.test.tencent;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;

public class ChatHistoryCrawler {

	private static String base = "http://qun.qq.com/air/24683344/chatlog/index/type/0/st/20090625/ap/7007/u//p/7015?w=n&_=0.17387192734869472";
	
	public static String crawler(String url){
		HttpClient http = new HttpClient();
//		Cookie cookie = new Cookie();
//		cookie.setValue("pvid=1645944102; flv=10.0 r22; avid=rGk826r3oP0YuF/YFFYWcghyyYvkTC7ROqfsSvptztwVF4wnjuUXjeyXCg8doatL1LdcJQ44Ufs=; r_cookie=952419147580; pt2gguin=o0582673485; ptcz=a5280fd0d1b0b3a0ffdefd047f4dacc5baee070d432754d119ebe661c6f19e54; o_cookie=582673485; webqq_setting=1343750144; pgv=ssid=s8682226506; verifysession=606a78ac55a37d975ff763afd4b64d667f444fc750a8ac00d26e62d9221f050e169cffa05d80ee8f; uin=o0582673485; skey=@c39zf296H; luin=o0582673485; lskey=25f1564d646109e2a80d4ff66572824abb824e8bcc2a8ca38cf4ef3af02b8a18ef9af2288e75aeb4; gkey=EWTuX7oGUbqmsD%2FaC3uyecQImna%2FPYYvQbLojYADFKTZMUZJjRGb5On6vOiRKVbdCSdmh9KaLbs%3D");
//		http.getState().addCookie(cookie);
		GetMethod get = new GetMethod(url);
		try {
			get.addRequestHeader("Host", "qun.qq.com");
			get.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)");
			get.addRequestHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			get.addRequestHeader("Accept-Language","zh-cn,zh;q=0.5");
			get.addRequestHeader("Accept-Encoding","gzip,deflate");
			get.addRequestHeader("Accept-Charset","gb2312,utf-8;q=0.7,*;q=0.7");
			get.addRequestHeader("Keep-Alive","300");
			get.addRequestHeader("Connection","keep-alive");
			//get.addRequestHeader("Cookie","pvid=1645944102; flv=10.0 r22; avid=rGk826r3oP0YuF/YFFYWcghyyYvkTC7ROqfsSvptztwVF4wnjuUXjeyXCg8doatL1LdcJQ44Ufs=; r_cookie=952419147580; pt2gguin=o0582673485; ptcz=a5280fd0d1b0b3a0ffdefd047f4dacc5baee070d432754d119ebe661c6f19e54; o_cookie=582673485; webqq_setting=1343750144; pgv=ssid=s8682226506; verifysession=606a78ac55a37d975ff763afd4b64d667f444fc750a8ac00d26e62d9221f050e169cffa05d80ee8f; uin=o0582673485; skey=@c39zf296H; luin=o0582673485; lskey=25f1564d646109e2a80d4ff66572824abb824e8bcc2a8ca38cf4ef3af02b8a18ef9af2288e75aeb4; gkey=EWTuX7oGUbqmsD%2FaC3uyecQImna%2FPYYvQbLojYADFKTZMUZJjRGb5On6vOiRKVbdCSdmh9KaLbs%3D");
			
			get.addRequestHeader("Cookie","pvid=1645944102; flv=10.0 r22; avid=rGk826r3oP0YuF/YFFYWcghyyYvkTC7ROqfsSvptztwVF4wnjuUXjeyXCg8doatL1LdcJQ44Ufs=; r_cookie=952419147580; pt2gguin=o0582673485; ptcz=a5280fd0d1b0b3a0ffdefd047f4dacc5baee070d432754d119ebe661c6f19e54; o_cookie=582673485; webqq_setting=1343750144; pgv=ssid=s8682226506; verifysession=606a78ac55a37d975ff763afd4b64d667f444fc750a8ac00d26e62d9221f050e169cffa05d80ee8f; uin=o0582673485; skey=@c39zf296H; luin=o0582673485; lskey=25f1564d646109e2a80d4ff66572824abb824e8bcc2a8ca38cf4ef3af02b8a18ef9af2288e75aeb4; gkey=EWTuX7oGUbqmsD%2FaC3uyecQImna%2FPYYvQbLojYADFKTZMUZJjRGb5On6vOiRKVbdCSdmh9KaLbs%3D");
			
			int er = http.executeMethod(get);
			if (er == 200) {
				//System.out.println(get.getResponseContentLength());
				//System.out.println("response.charset() = " + get.getResponseCharSet());
				String html = get.getResponseBodyAsString();
				//System.out.println(new String(html.getBytes("ISO-8859-1"),"UTF-8"));
				System.out.println(url);
				System.out.println(html.getBytes().length + " bytes, " + (html.getBytes().length/1024) + " kb.");
				return new String(html.getBytes("ISO-8859-1"),"UTF-8");
			} else {
				return "";
			}
		} catch (HttpException e) {
			e.printStackTrace();
			return "";
		} catch (IOException e) {
			e.printStackTrace();
			return "";
		} finally {
			get.releaseConnection();
		}
	}
	
	public static void crawlerJSTest(){
		HttpClient http = new HttpClient();
		GetMethod get = new GetMethod("http://www.google.cn");
		try {
			get.addRequestHeader("Host", "qun.qq.com");
			get.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)");
			get.addRequestHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			get.addRequestHeader("Accept-Language","zh-cn,zh;q=0.5");
			get.addRequestHeader("Accept-Encoding","gzip,deflate");
			get.addRequestHeader("Accept-Charset","gb2312,utf-8;q=0.7,*;q=0.7");
			get.addRequestHeader("Keep-Alive","300");
			get.addRequestHeader("Connection","keep-alive");
			int er = http.executeMethod(get);
			if (er == 200) {
				//System.out.println(get.getResponseContentLength());
				System.out.println("response.charset() = " + get.getResponseCharSet());
				String html = get.getResponseBodyAsString();
				//System.out.println(html);
				System.out.println(new String(html.getBytes("gb2312"),"UTF-8"));
				System.out.println(html.getBytes().length + " bytes, " + (html.getBytes().length/1024) + " kb.");
				
			}
		} catch (HttpException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			get.releaseConnection();
		}
	}
	
	/**
	 * @param args
	 * @throws IOException 
	 * @throws HttpException 
	 * @throws  
	 */
	public static void main(String[] args) {
		crawlerJSTest();
		if(true)return;
		int page = 3120;
		while(true){
			if(page>7087){
				break;
			}
			String url = "http://qun.qq.com/air/24683344/chatlog/index/type/0/st/20090625/ap/7015/u//p/"+page+"?w=n&_=0.17387192734869472";
			String result = crawler(url);
			if(result == null || result.equals("")){
				
			}else{
				PrintWriter out = null;
				try {
					out = new PrintWriter(new BufferedWriter(new FileWriter("E:\\QQGroupChatHistory\\24683344\\"+page+".html")));
					out.print(result);
					out.flush();
				} catch (IOException e) {
					e.printStackTrace();
				} finally {
					out.close();
				}
				
			}
			page++;
			try {
				Thread.sleep(1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}

}
